// -*- c -*-
//
// %CopyrightBegin%
//
// Copyright Ericsson AB 2017-2024. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// %CopyrightEnd%
//

%if ARCH_64
BS_SAFE_MUL(A, B, Fail, Dst) {
    Uint a = $A;
    Uint b = $B;
    Uint res;
#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
    if (__builtin_mul_overflow(a, b, &res)) {
        $Fail;
    }
#else
    res = a * b;
    if (res / b != a) {
        $Fail;
    }
#endif
    $Dst = res;
}

BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
    if (is_small($Bits)) {
        Uint uint_size;
        Sint signed_size = signed_val($Bits);
        if (signed_size < 0) {
            $Fail;
        }
        uint_size = (Uint) signed_size;
        $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst);
    } else {
        /*
         * On a 64-bit architecture, the size of any binary
         * that would fit in the memory fits in a small.
         */
        $Fail;
    }
}

BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
    if (is_small($Bits)) {
        Uint uint_size;
        Sint signed_size = signed_val($Bits);
        if (signed_size < 0) {
            $Fail;
        }
        uint_size = (Uint) signed_size;
        $Dst = uint_size * $Unit;
    } else {
        /*
         * On a 64-bit architecture, the size of any binary
         * that would fit in the memory fits in a small.
         */
        $Fail;
    }
}
%else
BS_SAFE_MUL(A, B, Fail, Dst) {
    Uint64 res = (Uint64)($A) * (Uint64)($B);
    if ((res >> (8*sizeof(Uint))) != 0) {
        $Fail;
    }
    $Dst = res;
}

BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
    Sint signed_size;
    Uint uint_size;
    Uint temp_bits;

    if (is_small($Bits)) {
        signed_size = signed_val($Bits);
        if (signed_size < 0) {
            $Fail;
        }
        uint_size = (Uint) signed_size;
    } else {
        if (!term_to_Uint($Bits, &temp_bits)) {
            $Fail;
        }
        uint_size = temp_bits;
    }
    $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst);
}

BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
    Sint signed_size;
    Uint uint_size;
    Uint temp_bits;

    if (is_small($Bits)) {
        signed_size = signed_val($Bits);
        if (signed_size < 0) {
            $Fail;
        }
        uint_size = (Uint) signed_size;
    } else {
        if (!term_to_Uint($Bits, &temp_bits)) {
            $Fail;
        }
        uint_size = temp_bits;
    }
    $Dst = uint_size * $Unit;
}
%endif

TEST_BIN_VHEAP(VNh, Nh, Live) {
    Uint need = $Nh;
    if ((E - HTOP < (need + S_RESERVED)) ||
        (MSO(c_p).overhead + $VNh >= c_p->bin_vheap_sz)) {
        $GC_SWAPOUT();
        PROCESS_MAIN_CHK_LOCKS(c_p);
        FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
        PROCESS_MAIN_CHK_LOCKS(c_p);
        SWAPIN;
        $MAYBE_EXIT_AFTER_GC();
    }
    HEAP_SPACE_VERIFIED(need);
}

i_bs_get_binary_all2 := i_bs_get_binary_all2.fetch.execute;

i_bs_get_binary_all2.head() {
    Eterm context;
}

i_bs_get_binary_all2.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_binary_all2.execute(Fail, Live, Unit, Dst) {
    ErlSubBits *sb;
    Eterm _result;

    $GC_TEST_PRESERVE(BUILD_SUB_BITSTRING_HEAP_NEED, $Live, context);
    sb = (ErlSubBits*)bitstring_val(context);

    if (((sb->end - sb->start) % $Unit) == 0) {
        LIGHT_SWAPOUT;
        _result = erts_bs_get_binary_all_2(c_p, sb);
        LIGHT_SWAPIN;
        HEAP_SPACE_VERIFIED(0);
        ASSERT(is_value(_result));
        $REFRESH_GEN_DEST();
        $Dst = _result;
    } else {
	HEAP_SPACE_VERIFIED(0);
	$FAIL($Fail);
    }
}
i_bs_get_binary2 := i_bs_get_binary2.fetch.execute;

i_bs_get_binary2.head() {
    Eterm context;
}

i_bs_get_binary2.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_binary2.execute(Fail, Live, Sz, Flags, Dst) {
    ErlSubBits *sb;
    Eterm _result;
    Uint _size;
    $BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
    $GC_TEST_PRESERVE(BUILD_SUB_BITSTRING_HEAP_NEED, $Live, context);
    sb = (ErlSubBits*)bitstring_val(context);
    LIGHT_SWAPOUT;
    _result = erts_bs_get_binary_2(c_p, _size, $Flags, sb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(_result)) {
        $FAIL($Fail);
    } else {
        $REFRESH_GEN_DEST();
        $Dst = _result;
    }
}

i_bs_get_binary_imm2 := i_bs_get_binary_imm2.fetch.execute;

i_bs_get_binary_imm2.head() {
    Eterm context;
}

i_bs_get_binary_imm2.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_binary_imm2.execute(Fail, Live, Sz, Flags, Dst) {
    ErlSubBits *sb;
    Eterm _result;
    $GC_TEST_PRESERVE(BUILD_SUB_BITSTRING_HEAP_NEED, $Live, context);
    sb = (ErlSubBits*)bitstring_val(context);
    LIGHT_SWAPOUT;
    _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, sb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(_result)) {
        $FAIL($Fail);
    } else {
        $REFRESH_GEN_DEST();
        $Dst = _result;
    }
}
i_bs_get_float2 := i_bs_get_float2.fetch.execute;

i_bs_get_float2.head() {
    Eterm context;
}

i_bs_get_float2.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_float2.execute(Fail, Live, Sz, Flags, Dst) {
    ErlSubBits *sb;
    Eterm _result;
    Sint _size;

    if (!is_small($Sz) || (_size = unsigned_val($Sz)) > 64) {
        $FAIL($Fail);
    }
    _size *= (($Flags) >> 3);
    $GC_TEST_PRESERVE(FLOAT_SIZE_OBJECT, $Live, context);
    sb = (ErlSubBits*)bitstring_val(context);
    LIGHT_SWAPOUT;
    _result = erts_bs_get_float_2(c_p, _size, ($Flags), sb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(_result)) {
        $FAIL($Fail);
    } else {
        $REFRESH_GEN_DEST();
        $Dst = _result;
    }
}

i_bs_skip_bits2 := i_bs_skip_bits2.fetch.execute;

i_bs_skip_bits2.head() {
    Eterm context, bits;
}

i_bs_skip_bits2.fetch(Ctx, Bits) {
    context = $Ctx;
    bits = $Bits;
}

i_bs_skip_bits2.execute(Fail, Unit) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    size_t new_offset;
    Uint _size;

    $BS_GET_FIELD_SIZE(bits, $Unit, $FAIL($Fail), _size);
    new_offset = sb->start + _size;
    if (new_offset <= sb->end) {
        sb->start = new_offset;
    } else {
        $FAIL($Fail);
    }
}

i_bs_skip_bits_imm2(Fail, Ms, Bits) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ms);
    size_t new_offset;
    new_offset = sb->start + ($Bits);
    if (new_offset <= sb->end) {
        sb->start = new_offset;
    } else {
        $FAIL($Fail);
    }
}

i_new_bs_put_binary(Fail, Sz, Flags, Src) {
    Eterm sz = $Sz;
    Sint _size;
    $BS_GET_UNCHECKED_FIELD_SIZE(sz, (($Flags) >> 3), $BADARG($Fail), _size);
    c_p->fcalls = FCALLS;
    if (!erts_new_bs_put_binary(c_p, $Src, _size)) {
        $BADARG($Fail);
    }
    FCALLS = c_p->fcalls;
}

i_new_bs_put_binary_all := i_new_bs_put_binary_all.fetch.execute;

i_new_bs_put_binary_all.head() {
    Eterm src;
}

i_new_bs_put_binary_all.fetch(Src) {
    src = $Src;
}

i_new_bs_put_binary_all.execute(Fail, Unit) {
    c_p->fcalls = FCALLS;
    if (!erts_new_bs_put_binary_all(c_p, src, ($Unit))) {
        $BADARG($Fail);
    }
    FCALLS = c_p->fcalls;
}

i_new_bs_put_binary_imm(Fail, Sz, Src) {
    c_p->fcalls = FCALLS;
    if (!erts_new_bs_put_binary(c_p, ($Src), ($Sz))) {
        $BADARG($Fail);
    }
    FCALLS = c_p->fcalls;
}

i_new_bs_put_float(Fail, Sz, Flags, Src) {
    Eterm sz = $Sz;
    Eterm flags = $Flags;
    Sint _size;
    $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size);
    if (is_value(erts_new_bs_put_float(c_p, ($Src), _size, flags))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_float_imm(Fail, Sz, Flags, Src) {
    if (is_value(erts_new_bs_put_float(c_p, ($Src), ($Sz), ($Flags)))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_integer(Fail, Sz, Flags, Src) {
    Eterm sz = $Sz;
    Eterm flags = $Flags;
    Sint _size;
    $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size);
    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, flags))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_integer_imm := i_new_bs_put_integer_imm.fetch.execute;

i_new_bs_put_integer_imm.head() {
    Eterm src;
}

i_new_bs_put_integer_imm.fetch(Src) {
    src = $Src;
}

i_new_bs_put_integer_imm.execute(Fail, Sz, Flags) {
    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(src, ($Sz), ($Flags)))) {
        $BADARG($Fail);
    }
}

#
# i_bs_init*
#

i_bs_init_fail_heap := bs_init.fail_heap.verify.execute;
i_bs_init_fail := bs_init.fail.verify.execute;
i_bs_init := bs_init.plain.execute;
i_bs_init_heap := bs_init.heap.execute;

bs_init.head() {
    Eterm BsOp1;
    Eterm BsOp2;
}

bs_init.fail_heap(Size, HeapAlloc) {
    BsOp1 = $Size;
    BsOp2 = $HeapAlloc;
}

bs_init.fail(Size) {
    BsOp1 = $Size;
    BsOp2 = 0;
}

bs_init.plain(Size) {
    BsOp1 = $Size;
    BsOp2 = 0;
}

bs_init.heap(Size, HeapAlloc) {
    BsOp1 = $Size;
    BsOp2 = $HeapAlloc;
}

bs_init.verify(Fail) {
    if (is_small(BsOp1)) {
        Sint size = signed_val(BsOp1);
        if (size < 0) {
            $BADARG($Fail);
        }
        BsOp1 = (Eterm) size;
    } else {
        Uint bytes;

        if (!term_to_Uint(BsOp1, &bytes)) {
            c_p->freason = bytes;
            $FAIL_HEAD_OR_BODY($Fail);
        }
        if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
            $SYSTEM_LIMIT($Fail);
        }
        BsOp1 = (Eterm) bytes;
    }
}

bs_init.execute(Live, Dst) {
    Uint num_bits, heap_extra;
    Eterm new_binary;

    erts_bin_offset = 0;

    num_bits = BsOp1 * CHAR_BIT;
    heap_extra = BsOp2;

    if (num_bits <= ERL_ONHEAP_BITS_LIMIT) {
        ErlHeapBits *hb;
        Uint bin_need;

        bin_need = heap_bits_size(num_bits);
        $GC_TEST(0, bin_need + heap_extra + ERL_SUB_BITS_SIZE, $Live);

        hb = (ErlHeapBits*)HTOP;
        HTOP += bin_need;

        hb->thing_word = header_heap_bits(num_bits);
        hb->size = num_bits;
        erts_current_bin = (byte *) hb->data;

        new_binary = make_bitstring(hb);
    } else {
        Binary* bptr;

        $TEST_BIN_VHEAP(NBYTES(num_bits) / sizeof(Eterm),
                        heap_extra + ERL_REFC_BITS_SIZE,
                        $Live);

        bptr = erts_bin_nrml_alloc(NBYTES(num_bits));
        erts_current_bin = (byte *) bptr->orig_bytes;

        LIGHT_SWAPOUT;

        new_binary = erts_wrap_refc_bitstring(&MSO(c_p).first,
                                              &MSO(c_p).overhead,
                                              &HEAP_TOP(c_p),
                                              bptr,
                                              erts_current_bin,
                                              0,
                                              num_bits);

        LIGHT_SWAPIN;
    }

    $Dst = new_binary;
}

#
# i_bs_init_bits*
#

i_bs_init_bits           := bs_init_bits.plain.execute;
i_bs_init_bits_heap      := bs_init_bits.heap.execute;
i_bs_init_bits_fail      := bs_init_bits.fail.verify.execute;
i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute;

bs_init_bits.head() {
    Eterm num_bits_term;
    Uint num_bits;
    Uint alloc;
}

bs_init_bits.plain(NumBits) {
    num_bits = $NumBits;
    alloc = 0;
}

bs_init_bits.heap(NumBits, Alloc) {
    num_bits = $NumBits;
    alloc = $Alloc;
}

bs_init_bits.fail(NumBitsTerm) {
    num_bits_term = $NumBitsTerm;
    alloc = 0;
}

bs_init_bits.fail_heap(NumBitsTerm, Alloc) {
    num_bits_term = $NumBitsTerm;
    alloc = $Alloc;
}

bs_init_bits.verify(Fail) {
    if (is_small(num_bits_term)) {
        Sint size = signed_val(num_bits_term);
        if (size < 0) {
            $BADARG($Fail);
        }
        num_bits = (Uint) size;
    } else {
        Uint bits;

        if (!term_to_Uint(num_bits_term, &bits)) {
            c_p->freason = bits;
            $FAIL_HEAD_OR_BODY($Fail);
        }
        num_bits = (Uint) bits;
    }
}

bs_init_bits.execute(Live, Dst) {
     Eterm new_binary;

     if (num_bits <= ERL_ONHEAP_BITS_LIMIT) {
        alloc += heap_bits_size(num_bits);
     } else {
        alloc += ERL_REFC_BITS_SIZE;
     }

     erts_bin_offset = 0;

     /* num_bits = Number of bits to build
      * num_bytes = Number of bytes to allocate in the binary
      * alloc = Total number of words to allocate on heap
      * Operands: NotUsed NotUsed Dst
      */
     if (num_bits <= ERL_ONHEAP_BITS_LIMIT) {
        ErlHeapBits *hb;

        $test_heap(alloc, $Live);

        hb = (ErlHeapBits*) HTOP;
        HTOP += heap_bits_size(num_bits);
        hb->thing_word = header_heap_bits(num_bits);

        hb->size = num_bits;

        erts_current_bin = (byte*)hb->data;
        new_binary = make_bitstring(hb);
     } else {
        Binary *bptr;

        $TEST_BIN_VHEAP(NBYTES(num_bits) / sizeof(Eterm),
                        alloc + ERL_REFC_BITS_SIZE,
                        $Live);

        bptr = erts_bin_nrml_alloc(NBYTES(num_bits));
        erts_current_bin = (byte *) bptr->orig_bytes;

        LIGHT_SWAPOUT;

        new_binary = erts_wrap_refc_bitstring(&MSO(c_p).first,
                                              &MSO(c_p).overhead,
                                              &HEAP_TOP(c_p),
                                              bptr,
                                              erts_current_bin,
                                              0,
                                              num_bits);

        LIGHT_SWAPIN;
     }

     HEAP_SPACE_VERIFIED(0);
     $Dst = new_binary;
}

bs_add(Fail, Src1, Src2, Unit, Dst) {
    Eterm Op1 = $Src1;
    Eterm Op2 = $Src2;
    Uint unit = $Unit;

    if (is_both_small(Op1, Op2)) {
        Sint Arg1 = signed_val(Op1);
        Sint Arg2 = signed_val(Op2);

        if (Arg1 >= 0 && Arg2 >= 0) {
            $BS_SAFE_MUL(Arg2, unit, $SYSTEM_LIMIT($Fail), Op1);
            Op1 += Arg1;

#ifdef ARCH_32
        store_bs_add_result:
#endif
            if (Op1 <= MAX_SMALL) {
                Op1 = make_small(Op1);
            } else {
                /*
                 * May generate a heap fragment, but in this
                 * particular case it is OK, since the value will be
                 * stored into an x register (the GC will scan x
                 * registers for references to heap fragments) and
                 * there is no risk that value can be stored into a
                 * location that is not scanned for heap-fragment
                 * references (such as the heap).
                 */
                SWAPOUT;
                Op1 = erts_make_integer(Op1, c_p);
                HTOP = HEAP_TOP(c_p);
            }
            $Dst = Op1;
            $NEXT0();
        }
        $BADARG($Fail);
    } else {
#ifdef ARCH_64
        /* The size must fit into a small on 64-bit platforms, which can't
         * happen if either operand is a bignum. */
        if (((is_small(Op1) && signed_val(Op1) >= 0) ||
             (is_big(Op1) && !big_sign(Op1))) &&
            ((is_small(Op2) && signed_val(Op2) >= 0) ||
             (is_big(Op2) && !big_sign(Op2)))) {
            $SYSTEM_LIMIT($Fail);
        } else {
            $BADARG($Fail);
        }
#else
        Uint a;
        Uint b;
        Uint c;

        /*
         * Now we know that one of the arguments is
         * not a small. We must convert both arguments
         * to Uints and check for errors at the same time.
         *
         * Error checking is tricky.
         *
         * If one of the arguments is not numeric or
         * not positive, the error reason is BADARG.
         *
         * Otherwise if both arguments are numeric,
         * but at least one argument does not fit in
         * an Uint, the reason is SYSTEM_LIMIT.
         */

        if (!term_to_Uint(Op1, &a)) {
            if (a == BADARG) {
                c_p->freason = a;
                $FAIL_HEAD_OR_BODY($Fail);
            }
            if (!term_to_Uint(Op2, &b)) {
                c_p->freason = b;
                $FAIL_HEAD_OR_BODY($Fail);
            }
            $SYSTEM_LIMIT($Fail);
        } else if (!term_to_Uint(Op2, &b)) {
            c_p->freason = b;
            $FAIL_HEAD_OR_BODY($Fail);
        }

        /*
         * The arguments are now correct and stored in a and b.
         */

        $BS_SAFE_MUL(b, unit, $SYSTEM_LIMIT($Fail), c);
        Op1 = a + c;
        if (Op1 < a) {
            /*
             * If the result is less than one of the
             * arguments, there must have been an overflow.
             */
            $SYSTEM_LIMIT($Fail);
        }
        goto store_bs_add_result;
#endif
    }
    /* No fallthrough */
    ASSERT(0);
}

bs_put_string(Len, Ptr) {
    erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) $Ptr, $Len));
}

i_bs_append(Fail, ExtraHeap, Live, Unit, Size, Dst) {
    Uint live = $Live;
    Uint res;

    HEAVY_SWAPOUT;
    reg[live] = x(SCRATCH_X_REG);
    res = erts_bs_append(c_p, reg, live, $Size, $ExtraHeap, $Unit);
    HEAVY_SWAPIN;
    if (is_non_value(res)) {
        $MAYBE_EXIT_AFTER_GC();

        /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */
        $FAIL_HEAD_OR_BODY($Fail);
    }
    $Dst = res;
}

i_bs_private_append(Fail, Unit, Size, Src, Dst) {
    Eterm res;

    c_p->fcalls = FCALLS;
    res = erts_bs_private_append(c_p, $Src, $Size, $Unit);
    if (is_non_value(res)) {
        /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */
        $FAIL_HEAD_OR_BODY($Fail);
    }
    FCALLS = c_p->fcalls;
    $Dst = res;
}

bs_init_writable() {
    HEAVY_SWAPOUT;
    x(0) = erts_bs_init_writable(c_p, x(0));
    HEAVY_SWAPIN;
}

i_bs_utf8_size(Src, Dst) {
    Eterm arg = $Src;
    Eterm result;

    /*
     * Calculate the number of bytes needed to encode the source
     * operand to UTF-8. If the source operand is invalid (e.g. wrong
     * type or range) we return a nonsense integer result (0 or 4). We
     * can get away with that because we KNOW that bs_put_utf8 will do
     * full error checking.
     */

    if (arg < make_small(0x80UL)) {
        result = make_small(1);
    } else if (arg < make_small(0x800UL)) {
        result = make_small(2);
    } else if (arg < make_small(0x10000UL)) {
        result = make_small(3);
    } else {
        result = make_small(4);
    }
    $Dst = result;
}

i_bs_put_utf8(Fail, Src) {
    if (!erts_bs_put_utf8(ERL_BITS_ARGS_1($Src))) {
        $BADARG($Fail);
    }
}

i_bs_utf16_size(Src, Dst) {
    Eterm arg = $Src;
    Eterm result = make_small(2);

    /*
     * Calculate the number of bytes needed to encode the source
     * operarand to UTF-16. If the source operand is invalid (e.g. wrong
     * type or range) we return a nonsense integer result (2 or 4). We
     * can get away with that because we KNOW that bs_put_utf16 will do
     * full error checking.
     */

    if (arg >= make_small(0x10000UL)) {
        result = make_small(4);
    }
    $Dst = result;
}

i_bs_put_utf16(Fail, Flags, Src) {
    if (!erts_bs_put_utf16(ERL_BITS_ARGS_2($Src, $Flags))) {
        $BADARG($Fail);
    }
}

// Validate a value about to be stored in a binary.
i_bs_validate_unicode(Fail, Src) {
    Eterm val = $Src;

    /*
     * There is no need to untag the integer, but it IS necessary
     * to make sure it is small (if the term is a bignum, it could
     * slip through the test, and there is no further test that
     * would catch it, since bit syntax construction silently masks
     * too big numbers).
     */
    if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
        (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
        $BADARG($Fail);
    }
}

// Validate a value that has been matched out.
i_bs_validate_unicode_retract(Fail, Src, Ms) {
    /*
     * There is no need to untag the integer, but it IS necessary
     * to make sure it is small (a bignum pointer could fall in
     * the valid range).
     */

    Eterm i = $Src;
    if (is_not_small(i) || i > make_small(0x10FFFFUL) ||
        (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) {
        ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ms);

        /* Invalid value. Retract the position in the binary. */
        sb->start -= 32;
        $BADARG($Fail);
    }
}

BS_GET_TERM(Term, Dst) {
    $Dst = $Term;
    switch (loader_tag($Dst)) {
    case LOADER_X_REG:
        $Dst = x(loader_x_reg_index($Dst));
        break;
    case LOADER_Y_REG:
        $Dst = y(loader_y_reg_index($Dst));
        break;
    }
}

BS_LOAD_UNIT(Ptr, Dst) {
    $Dst = $Ptr[1];
}

BS_LOAD_FLAGS(Ptr, Dst) {
    $Dst = $Ptr[2];
}

BS_LOAD_SRC(Ptr, Dst) {
    $BS_GET_TERM($Ptr[3], $Dst);
}

BS_LOAD_STRING_SRC(Ptr, Dst) {
    $Dst = (byte *) $Ptr[3];
}

BS_LOAD_SIZE(Ptr, Dst) {
    $BS_GET_TERM($Ptr[4], $Dst);
}

BS_LOAD_FIXED_SIZE(Ptr, Dst) {
    $Dst = $Ptr[4];
}

// Implicitly uses the c_p and p variables (for convenience).
BS_FAIL_INFO(Fail, Reason, ErrorType, Value) {
    erts_prepare_bs_construct_fail_info(c_p, p, $Reason, $ErrorType, $Value);
    $FAIL_HEAD_OR_BODY($Fail);
}

// Implicitly uses the Size variable because of limitations of parsing in
// beam_makeops of nested macro call; a nested macro call can only have one
// argument.
BS_FAIL_INFO_SYSTEM_LIMIT(Fail) {
    $BS_FAIL_INFO($Fail, SYSTEM_LIMIT, am_size, Size);
}

i_bs_create_bin(Fail, Alloc, Live, Dst, N) {
    //| -no_prefetch
    int n = $N;
    const BeamInstr* p_start = $NEXT_INSTRUCTION;
    const BeamInstr* p_end = p_start + n;
    const BeamInstr* p;
    Uint alloc = $Alloc;
    Eterm new_binary;

    /* We count the total number of bits in an unsigned integer. To avoid
     * having to check for overflow when adding to `num_bits`, we ensure that
     * the signed size of each segment fits in a word. */
    Uint num_bits = 0;

    /* Calculate size of binary in bits. */
    for (p = p_start; p < p_end; p += BSC_NUM_ARGS) {
        Eterm Src;
        Eterm Size;
        Uint unit;
        Uint fixed_size;

        switch (p[0]) {
        case BSC_APPEND:
        case BSC_PRIVATE_APPEND:
            break;
        case BSC_BINARY_ALL:
            {
                Uint bit_size;

                $BS_LOAD_SRC(p, Src);
                if (is_not_bitstring(Src)) {
                    $BS_FAIL_INFO($Fail, BADARG, am_type, Src);
                }

                bit_size = bitstring_size(Src);
#ifndef ARCH_64
                if (bit_size >= ERTS_SINT_MAX) {
                    /* The size of the binary in bits will not fit in a 32-bit
                     * signed integer. */
                    $BS_FAIL_INFO($Fail, SYSTEM_LIMIT, am_binary, am_size);
                }
#endif
                num_bits += bit_size;
            }
            break;
        case BSC_BINARY_FIXED_SIZE:
        case BSC_FLOAT_FIXED_SIZE:
        case BSC_INTEGER_FIXED_SIZE:
            $BS_LOAD_FIXED_SIZE(p, fixed_size);
            num_bits += fixed_size;
            break;
        case BSC_STRING:
            $BS_LOAD_FIXED_SIZE(p, fixed_size);
            num_bits += fixed_size * 8;
            break;
        case BSC_BINARY:
        case BSC_FLOAT:
        case BSC_INTEGER:
            $BS_LOAD_UNIT(p, unit);
            $BS_LOAD_SIZE(p, Size);
            if (is_small(Size)) {
                Sint signed_size = signed_val(Size);
                Uint size;
                if (signed_size >= 0) {
                    $BS_SAFE_MUL(signed_size, unit, $BS_FAIL_INFO_SYSTEM_LIMIT($Fail), size);
                    if (size >> (sizeof(Uint) * 8 - 1) != 0) {
                        /* The signed size does not fit in a word. */
                        $BS_FAIL_INFO($Fail, SYSTEM_LIMIT, am_size, Size);
                    }
                    num_bits += size;
                } else {
                    $BS_FAIL_INFO($Fail, BADARG, am_size, Size);
                }
            } else {
#ifdef ARCH_64
                /* The size must fit in a small on 64-bit platforms. */
                if (is_big(Size)) {
                    if (!big_sign(Size)) {
                        $BS_FAIL_INFO($Fail, SYSTEM_LIMIT, am_size, Size);
                    } else {
                        $BS_FAIL_INFO($Fail, BADARG, am_size, Size);
                    }
                } else {
                    /* Not an integer. */
                    $BS_FAIL_INFO($Fail, BADARG, am_size, Size);
                }
#else
                Uint size;

                if (!term_to_Uint(Size, &size)) {
                    if (size == BADARG) {
                        /* Not an integer or a negative integer. Determine which. */
                        if (is_big(Size)) {
                            /* Negative integer. */
                            $BS_FAIL_INFO($Fail, BADARG, am_size, Size);
                        }
                        /* Not an integer. */
                        $BS_FAIL_INFO($Fail, BADARG, am_size, Size);
                    }
                    /* Huge positive integer. */
                    $BS_FAIL_INFO_SYSTEM_LIMIT($Fail);
                }
                $BS_SAFE_MUL(size, unit, $BS_FAIL_INFO_SYSTEM_LIMIT($Fail), size);
                if ((size >> 31) != 0) {
                    $BS_FAIL_INFO_SYSTEM_LIMIT($Fail);
                } else {
                    num_bits += size;
                }
#endif
            }
            break;
        case BSC_UTF8:
            {
                int num_bytes;

                /*
                 * Calculate the number of bits needed to encode the
                 * source operand to UTF-8. If the source operand is
                 * invalid (e.g. wrong type or range) we return a
                 * nonsense integer result (32). We can get away
                 * with that because we KNOW that full error checking
                 * will be done in the construction phase.
                 */

                $BS_LOAD_SRC(p, Src);
                if (Src < make_small(0x80UL)) {
                    num_bytes = 1;
                } else if (Src < make_small(0x800UL)) {
                    num_bytes = 2;
                } else if (Src < make_small(0x10000UL)) {
                    num_bytes = 3;
                } else {
                    num_bytes = 4;
                }
                num_bits += num_bytes * 8;
            }
            break;
        case BSC_UTF16:
            {
                int num_bytes = 2;

                /*
                 * Calculate the number of bits needed to encode the
                 * source operarand to UTF-16. If the source operand
                 * is invalid (e.g. wrong type or range) we return a
                 * nonsense integer result (16 or 32). We can get away
                 * with that because we KNOW that full error checking
                 * will be done in the construction phase.
                 */

                $BS_LOAD_SRC(p, Src);
                if (Src >= make_small(0x10000UL)) {
                    num_bytes = 4;
                }
                num_bits += num_bytes * 8;
            }
            break;
        case BSC_UTF32:
            $BS_LOAD_SRC(p, Src);

            /*
             * There is no need to untag the integer, but it IS
             * necessary to make sure it is small (if the term is a
             * bignum, it could slip through the test, and there is no
             * further test that would catch it, since bit syntax
             * construction silently masks too big numbers).
             */
            if (is_not_small(Src) || Src > make_small(0x10FFFFUL) ||
                (make_small(0xD800UL) <= Src && Src <= make_small(0xDFFFUL))) {
                $BS_FAIL_INFO($Fail, BADARG, am_type, Src);
            }
            num_bits += 4 * 8;
            break;
        default:
            ASSERT(0);
            break;
        }
    }

    /* Allocate binary. */
    p = p_start;
    if (p[0] == BSC_APPEND) {
        Uint live = $Live;
        Uint unit;
        Eterm Src;

        $BS_LOAD_UNIT(p, unit);
        $BS_LOAD_SRC(p, Src);
        HEAVY_SWAPOUT;
        reg[live] = Src;
        new_binary = erts_bs_append_checked(c_p, reg, live, num_bits, alloc, unit);
        HEAVY_SWAPIN;
        if (is_non_value(new_binary)) {
            $MAYBE_EXIT_AFTER_GC();
            $BS_FAIL_INFO($Fail, c_p->freason, c_p->fvalue, reg[live]);
        }
        p_start += BSC_NUM_ARGS;
    } else if (p[0] == BSC_PRIVATE_APPEND) {
        Uint unit;
        Eterm Src;

        $test_heap(alloc, $Live);

        $BS_LOAD_UNIT(p, unit);
        $BS_LOAD_SRC(p, Src);

        new_binary = erts_bs_private_append_checked(c_p, Src, num_bits, unit);

        if (is_non_value(new_binary)) {
            $BS_FAIL_INFO($Fail, c_p->freason, c_p->fvalue, Src);
        }
        p_start += BSC_NUM_ARGS;
    } else {
        if (num_bits <= ERL_ONHEAP_BITS_LIMIT) {
            alloc += heap_bits_size(num_bits);
        } else {
            alloc += ERL_REFC_BITS_SIZE;
        }

        /* num_bits = Number of bits to build
         * alloc = Total number of words to allocate on heap
         */
        erts_bin_offset = 0;
        if (num_bits <= ERL_ONHEAP_BITS_LIMIT) {
            ErlHeapBits *hb;

            $test_heap(alloc, $Live);
            hb = (ErlHeapBits *) HTOP;
            HTOP += heap_bits_size(num_bits);
            hb->thing_word = header_heap_bits(num_bits);
            hb->size = num_bits;
            erts_current_bin = (byte *) hb->data;
            new_binary = make_bitstring(hb);
        } else {
            Binary* bptr;

            $TEST_BIN_VHEAP(NBYTES(num_bits) / sizeof(Eterm),
                            alloc + ERL_REFC_BITS_SIZE,
                            $Live);

            bptr = erts_bin_nrml_alloc(NBYTES(num_bits));
            erts_current_bin = (byte *)bptr->orig_bytes;

            LIGHT_SWAPOUT;

            new_binary = erts_wrap_refc_bitstring(&MSO(c_p).first,
                                                  &MSO(c_p).overhead,
                                                  &HEAP_TOP(c_p),
                                                  bptr,
                                                  erts_current_bin,
                                                  0,
                                                  num_bits);

            LIGHT_SWAPIN;
        }

        HEAP_SPACE_VERIFIED(0);
    }

    c_p->fcalls = FCALLS;

    /* Construct the segments. */
    for (p = p_start; p < p_end; p += BSC_NUM_ARGS) {
        Eterm Src;
        Eterm Size;
        Eterm flags;
        Eterm unit;
        Sint _size;

        if(p[0] == BSC_STRING) {
            byte* string;
            $BS_LOAD_STRING_SRC(p, string);
            $BS_LOAD_FIXED_SIZE(p, Size);
            erts_new_bs_put_string(ERL_BITS_ARGS_2(string, Size));
            continue;
        }

        $BS_LOAD_SRC(p, Src);

        switch (p[0]) {
        case BSC_BINARY_ALL:
            $BS_LOAD_UNIT(p, unit);
            if (!erts_new_bs_put_binary_all(c_p, Src, unit)) {
                $BS_FAIL_INFO($Fail, BADARG, am_unit, Src);
            }
            break;
        case BSC_BINARY:
            $BS_LOAD_UNIT(p, unit);
            $BS_LOAD_FLAGS(p, flags);
            $BS_LOAD_SIZE(p, Size);
            $BS_GET_UNCHECKED_FIELD_SIZE(Size, unit, $BADARG($Fail), _size);
            if (!erts_new_bs_put_binary(c_p, Src, _size)) {
                Eterm reason = is_bitstring(Src) ? am_short : am_type;
                $BS_FAIL_INFO($Fail, BADARG, reason, Src);
            }
            break;
        case BSC_BINARY_FIXED_SIZE:
            $BS_LOAD_FIXED_SIZE(p, Size);
            if (!erts_new_bs_put_binary(c_p, Src, Size)) {
                Eterm reason = is_bitstring(Src) ? am_short : am_type;
                $BS_FAIL_INFO($Fail, BADARG, reason, Src);
            }
            break;
        case BSC_FLOAT:
            $BS_LOAD_UNIT(p, unit);
            $BS_LOAD_FLAGS(p, flags);
            $BS_LOAD_SIZE(p, Size);
            $BS_GET_UNCHECKED_FIELD_SIZE(Size, unit, $BADARG($Fail), _size);
            Src = erts_new_bs_put_float(c_p, Src, _size, flags);
            if (is_value(Src)) {
                $BS_FAIL_INFO($Fail, BADARG, c_p->fvalue, Src);
            }
            break;
        case BSC_FLOAT_FIXED_SIZE:
            $BS_LOAD_FLAGS(p, flags);
            $BS_LOAD_FIXED_SIZE(p, Size);
            Src = erts_new_bs_put_float(c_p, Src, Size, flags);
            if (is_value(Src)) {
                $BS_FAIL_INFO($Fail, BADARG, c_p->fvalue, Src);
            }
            break;
        case BSC_INTEGER:
            {
                Sint _size;

                $BS_LOAD_UNIT(p, unit);
                $BS_LOAD_FLAGS(p, flags);
                $BS_LOAD_SIZE(p, Size);
                $BS_GET_UNCHECKED_FIELD_SIZE(Size, unit, $BADARG($Fail), _size);
                if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(Src, _size, flags))) {
                    $BS_FAIL_INFO($Fail, BADARG, am_type, Src);
                }
            }
            break;
        case BSC_INTEGER_FIXED_SIZE:
        case BSC_UTF32:
            $BS_LOAD_FLAGS(p, flags);
            $BS_LOAD_FIXED_SIZE(p, Size);
            if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(Src, Size, flags))) {
                $BS_FAIL_INFO($Fail, BADARG, am_type, Src);
            }
            break;
        case BSC_UTF8:
            if (!erts_bs_put_utf8(ERL_BITS_ARGS_1(Src))) {
                $BS_FAIL_INFO($Fail, BADARG, am_type, Src);
            }
            break;
        case BSC_UTF16:
            $BS_LOAD_FLAGS(p, flags);
            $BS_LOAD_SRC(p, Src);
            if (!erts_bs_put_utf16(ERL_BITS_ARGS_2(Src, flags))) {
                $BS_FAIL_INFO($Fail, BADARG, am_type, Src);
            }
            break;
        default:
            ASSERT(0);
            break;
        }
    }

    FCALLS = c_p->fcalls;

    /* Return the resulting binary. */
    $REFRESH_GEN_DEST();
    $Dst = new_binary;
    I += n;
}


//
// Matching of binaries.
//

bs_test_zero_tail2(Fail, Ctx) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    if (sb->end != sb->start) {
        $FAIL($Fail);
    }
}

bs_test_tail_imm2(Fail, Ctx, Offset) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    if (sb->end - sb->start != $Offset) {
        $FAIL($Fail);
    }
}

bs_test_unit(Fail, Ctx, Unit) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    if ((sb->end - sb->start) % $Unit) {
        $FAIL($Fail);
    }
}

bs_test_unit8(Fail, Ctx) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    if ((sb->end - sb->start) & 7) {
        $FAIL($Fail);
    }
}

i_bs_get_integer_8 := i_bs_get_integer_8.fetch.execute;

i_bs_get_integer_8.head() {
    Eterm context;
}

i_bs_get_integer_8.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_integer_8.execute(Fail, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Eterm _result;

    if (sb->end - sb->start < 8) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(sb->start) != 0) {
        _result = erts_bs_get_integer_2(c_p, 8, 0, sb);
    } else {
        _result = make_small(*(erl_sub_bits_get_base(sb) +
                                BYTE_OFFSET(sb->start)));
        sb->start += 8;
    }
    $Dst = _result;
}

i_bs_get_integer_16 := i_bs_get_integer_16.fetch.execute;

i_bs_get_integer_16.head() {
    Eterm context;
}

i_bs_get_integer_16.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_integer_16.execute(Fail, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Eterm _result;

    if (sb->end - sb->start < 16) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(sb->start) != 0) {
        _result = erts_bs_get_integer_2(c_p, 16, 0, sb);
    } else {
        _result = make_small(get_int16(erl_sub_bits_get_base(sb) +
                                        BYTE_OFFSET(sb->start)));
        sb->start += 16;
    }
    $Dst = _result;
}

%if ARCH_64
i_bs_get_integer_32 := i_bs_get_integer_32.fetch.execute;

i_bs_get_integer_32.head() {
    Eterm context;
}

i_bs_get_integer_32.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_integer_32.execute(Fail, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint32 _integer;

    if (sb->end - sb->start < 32) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(sb->start) != 0) {
        _integer = erts_bs_get_unaligned_uint32(sb);
    } else {
        _integer = get_int32(erl_sub_bits_get_base(sb) +
                              BYTE_OFFSET(sb->start));
    }
    sb->start += 32;
    $Dst = make_small(_integer);
}
%endif

i_bs_get_integer_imm := bs_get_integer.fetch.execute;
i_bs_get_integer_small_imm := bs_get_integer.fetch_small.execute;

bs_get_integer.head() {
    Eterm Ms, Sz;
}

bs_get_integer.fetch(Ctx, Size, Live) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    Uint wordsneeded;
    Ms = $Ctx;
    Sz = $Size;
    wordsneeded = 1+WSIZE(NBYTES(Sz));

    /* Check bits size before potential GC. We do not want a GC
     * and then realize we don't need the allocated space (if the
     * op fails).
     */
    if (sb->end - sb->start >= Sz) {
        $GC_TEST_PRESERVE(wordsneeded, $Live, Ms);
    }
}

bs_get_integer.fetch_small(Ctx, Size) {
    Ms = $Ctx;
    Sz = $Size;
}

bs_get_integer.execute(Fail, Flags, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(Ms);
    Eterm result;

    LIGHT_SWAPOUT;
    result = erts_bs_get_integer_2(c_p, Sz, $Flags, sb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $Dst = result;
}

i_bs_get_integer := i_bs_get_integer.fetch.execute;

i_bs_get_integer.head() {
    Eterm context;
}

i_bs_get_integer.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_integer.execute(Fail, Live, FlagsAndUnit, Sz, Dst) {
    ErlSubBits *sb;
    Uint flags;
    Uint size;
    Eterm result;

    flags = $FlagsAndUnit;
    $BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size);
    if (size >= SMALL_BITS) {
        Uint wordsneeded;
        /* Check bits size before potential gc.
         * We do not want a gc and then realize we don't need
         * the allocated space (i.e. if the op fails).
         *
         * Remember to re-acquire the match context after gc.
         */

        sb = (ErlSubBits*)bitstring_val(context);
        if (sb->end - sb->start < size) {
            $FAIL($Fail);
        }
        wordsneeded = 1+WSIZE(NBYTES((Uint) size));
        $GC_TEST_PRESERVE(wordsneeded, $Live, context);
        $REFRESH_GEN_DEST();
    }

    sb = (ErlSubBits*)bitstring_val(context);

    LIGHT_SWAPOUT;
    result = erts_bs_get_integer_2(c_p, size, flags, sb);
    LIGHT_SWAPIN;

    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $Dst = result;
}

i_bs_get_utf8 := i_bs_get_utf8.fetch.execute;

i_bs_get_utf8.head() {
    Eterm context;
}

i_bs_get_utf8.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_utf8.execute(Fail, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Eterm result;

    if (sb->end - sb->start < 8) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(sb->start) != 0) {
        result = erts_bs_get_utf8(sb);
    } else {
        byte b = *(erl_sub_bits_get_base(sb) + BYTE_OFFSET(sb->start));
        if (b < 128) {
            result = make_small(b);
            sb->start += 8;
        } else {
            result = erts_bs_get_utf8(sb);
        }
    }
    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $REFRESH_GEN_DEST();
    $Dst = result;
}

i_bs_get_utf16 := i_bs_get_utf16.fetch.execute;

i_bs_get_utf16.head() {
    Eterm context;
}

i_bs_get_utf16.fetch(Ctx) {
    context = $Ctx;
}

i_bs_get_utf16.execute(Fail, Flags, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Eterm result = erts_bs_get_utf16(sb, $Flags);

    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $REFRESH_GEN_DEST();
    $Dst = result;
}

i_bs_match_string(Ctx, Fail, Bits, Ptr) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    byte* bytes = (byte *) $Ptr;
    Uint size = $Bits;
    Uint offs;

    if (sb->end - sb->start < size) {
        $FAIL($Fail);
    }
    offs = BIT_OFFSET(sb->start);
    if (offs == 0 && TAIL_BITS(size) == 0) {
        if (sys_memcmp(bytes,
                       erl_sub_bits_get_base(sb) + BYTE_OFFSET(sb->start),
                       BYTE_SIZE(size))) {
            $FAIL($Fail);
        }
    } else if (erts_cmp_bits(bytes, 0,
                             erl_sub_bits_get_base(sb) + BYTE_OFFSET(sb->start),
                             BIT_OFFSET(sb->start),
                             size)) {
        $FAIL($Fail);
    }
    sb->start += size;
}

bs_get_tail := bs_get_tail.fetch.execute;

bs_get_tail.head() {
    Eterm context;
}

bs_get_tail.fetch(Src) {
    context = $Src;
}

bs_get_tail.execute(Dst, Live) {
    Eterm bin, *htop;
    ErlSubBits *sb;

    $GC_TEST_PRESERVE(BUILD_SUB_BITSTRING_HEAP_NEED, $Live, context);

    htop = HTOP;

    sb = (ErlSubBits*)bitstring_val(context);

    bin = erts_build_sub_bitstring(&htop,
                                   sb->orig & TAG_PTR_MASK__,
                                   (BinRef*)boxed_val(sb->orig),
                                   erl_sub_bits_get_base(sb),
                                   sb->start,
                                   sb->end - sb->start);
    HTOP = htop;

    $REFRESH_GEN_DEST();
    $Dst = bin;
}


%if ARCH_64

i_bs_start_match3_gp := i_bs_start_match3_gp.fetch.execute;

i_bs_start_match3_gp.head() {
    Eterm context;
}

i_bs_start_match3_gp.fetch(Src) {
    context = $Src;
}

i_bs_start_match3_gp.execute(Live, Fail, Dst, Pos) {
    Uint position, live;
    Eterm header;

    live = $Live;

    if (!is_boxed(context)) {
        $FAIL($Fail);
    }

    header = *boxed_val(context);

    if (is_bitstring_header(header)) {
        ErlSubBits *sb;
        int reuse = 0;

        if (header == HEADER_SUB_BITS) {
            sb = (ErlSubBits*)bitstring_val(context);
            reuse = erl_sub_bits_is_match_context(sb);
        }

        if (!reuse) {
            $GC_TEST_PRESERVE(ERL_SUB_BITS_SIZE, live, context);

            HEAP_TOP(c_p) = HTOP;
#ifdef DEBUG
            c_p->stop = E;        /* Needed for checking in HeapOnlyAlloc(). */
#endif
            sb = erts_bs_start_match_3(c_p, context);
            HTOP = HEAP_TOP(c_p);
            HEAP_SPACE_VERIFIED(0);

            $REFRESH_GEN_DEST();
            context = make_bitstring(sb);
        }

        position = sb->start;
        $Dst = context;
    } else {
        $FAIL($Fail);
    }

    ASSERT(IS_USMALL(0, position));
    $Pos = make_small(position);
}

bs_set_position(Ctx, Pos) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    sb->start = unsigned_val($Pos);
}

i_bs_get_position(Ctx, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);

    $Dst = make_small(sb->start);
}

%else

bs_set_position := bs_set_position.fetch.execute;

bs_set_position.head() {
    Eterm context, position;
}

bs_set_position.fetch(Ctx, Pos) {
    context = $Ctx;
    position = $Pos;
}

bs_set_position.execute() {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);

    if (ERTS_LIKELY(is_small(position))) {
        sb->start = unsigned_val(position);
    } else {
        ASSERT(is_big(position));
        sb->start = *BIG_V(big_val(position));
    }
}

bs_get_position := bs_get_position.fetch.execute;

bs_get_position.head() {
    Eterm context;
}

bs_get_position.fetch(Ctx) {
    context = $Ctx;
}

bs_get_position.execute(Dst, Live) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint position = sb->start;

    if (ERTS_LIKELY(IS_USMALL(0, position))) {
        $Dst = make_small(position);
    } else {
        Eterm *hp;

        $GC_TEST_PRESERVE(BIG_UINT_HEAP_SIZE, $Live, context);

        hp = HTOP;
        HTOP += BIG_UINT_HEAP_SIZE;

        *hp = make_pos_bignum_header(1);
        BIG_DIGIT(hp, 0) = position;

        $REFRESH_GEN_DEST();
        $Dst = make_big(hp);
    }
}

%endif

i_bs_start_match3 := i_bs_start_match3.fetch.execute;

i_bs_start_match3.head() {
    Eterm context;
}

i_bs_start_match3.fetch(Src) {
    context = $Src;
}

i_bs_start_match3.execute(Live, Fail, Dst) {
    Eterm header;
    Uint live;

    live = $Live;

    if (!is_boxed(context)) {
        $FAIL($Fail);
    }

    header = *boxed_val(context);

    if (is_bitstring_header(header)) {
        ErlSubBits *sb;
        int reuse = 0;

        if (header == HEADER_SUB_BITS) {
            sb = (ErlSubBits*)bitstring_val(context);
            reuse = erl_sub_bits_is_match_context(sb);
        }

        if (!reuse) {
            $GC_TEST_PRESERVE(ERL_SUB_BITS_SIZE, live, context);

            HEAP_TOP(c_p) = HTOP;
#ifdef DEBUG
            c_p->stop = E;        /* Needed for checking in HeapOnlyAlloc(). */
#endif
            sb = erts_bs_start_match_3(c_p, context);
            HTOP = HEAP_TOP(c_p);
            HEAP_SPACE_VERIFIED(0);

            $REFRESH_GEN_DEST();
            context = make_bitstring(sb);
        }

        $Dst = context;
    } else {
        $FAIL($Fail);
    }
}

//
// New instructions introduced in OTP 26 for matching of integers and
// binaries of fixed sizes follow.
//

//
// i_bs_ensure_bits Ctx Size Fail
//

i_bs_ensure_bits := i_bs_ensure_bits.fetch.execute;

i_bs_ensure_bits.head() {
    Eterm context;
}

i_bs_ensure_bits.fetch(Src) {
    context = $Src;
}

i_bs_ensure_bits.execute(NumBits, Fail) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint size = $NumBits;
    if (sb->end - sb->start < size) {
        $FAIL($Fail);
    }
}

//
// i_bs_ensure_bits_unit Ctx Size Unit Fail
//

i_bs_ensure_bits_unit := i_bs_ensure_bits_unit.fetch.execute;

i_bs_ensure_bits_unit.head() {
    Eterm context;
}

i_bs_ensure_bits_unit.fetch(Src) {
    context = $Src;
}

i_bs_ensure_bits_unit.execute(NumBits, Unit, Fail) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint size = $NumBits;
    Uint diff;

    if ((diff = sb->end - sb->start) < size) {
        $FAIL($Fail);
    }
    if ((diff - size) % $Unit != 0) {
        $FAIL($Fail);
    }
}

//
// i_bs_read_bits Ctx Size
// i_bs_ensure_bits_read Ctx Size Fail
//

i_bs_read_bits := i_bs_read_bits.fetch.execute;
i_bs_ensure_bits_read := i_bs_read_bits.fetch.ensure_bits.execute;

i_bs_read_bits.head() {
    ErlSubBits *sb;
    Uint size;
}

i_bs_read_bits.fetch(Src, NumBits) {
    sb = (ErlSubBits*)bitstring_val($Src);
    size = $NumBits;
}

i_bs_read_bits.ensure_bits(Fail) {
    if (sb->end - sb->start < size) {
        $FAIL($Fail);
    }
}

i_bs_read_bits.execute() {
    const byte *byte_ptr;
    Uint bit_offset = sb->start % 8;
    Uint num_bytes_to_read = (size + 7) / 8;
    Uint num_partial = size % 8;

    if ((num_partial == 0 && bit_offset != 0) ||
        (num_partial != 0 && bit_offset > 8 - num_partial)) {
        num_bytes_to_read++;
    }

    bitdata = 0;
    byte_ptr = erl_sub_bits_get_base(sb) + BYTE_OFFSET(sb->start);
    sb->start += size;
    switch (num_bytes_to_read) {
#ifdef ARCH_64
    case 9:
    case 8:
        bitdata = bitdata << 8 | *byte_ptr++;
    case 7:
        bitdata = bitdata << 8 | *byte_ptr++;
    case 6:
        bitdata = bitdata << 8 | *byte_ptr++;
    case 5:
        bitdata = bitdata << 8 | *byte_ptr++;
#else
    case 5:
#endif
    case 4:
        bitdata = bitdata << 8 | *byte_ptr++;
    case 3:
        bitdata = bitdata << 8 | *byte_ptr++;
    case 2:
        bitdata = bitdata << 8 | *byte_ptr++;
    case 1:
        bitdata = bitdata << 8 | *byte_ptr++;
    }

    if (num_bytes_to_read <= sizeof(Uint)) {
        bitdata <<= 8 * (sizeof(Uint) - num_bytes_to_read) + bit_offset;
    } else {
        bitdata <<= bit_offset;
        bitdata = bitdata | ((*byte_ptr << bit_offset) >> 8);
    }
}

// i_bs_eq Fail Size Value
i_bs_eq(Fail, NumBits, Value) {
    Uint size = $NumBits;
    Eterm result;

    result = bitdata >> (8 * sizeof(Uint) - size);
    bitdata <<= size;
    if (result != $Value) {
        $FAIL($Fail);
    }
}

// i_bs_extract_integer Size Dst
i_bs_extract_integer(NumBits, Dst) {
    Uint size = $NumBits;
    Eterm result;

    result = bitdata >> (8 * sizeof(Uint) - size);
    result = make_small(result);
    bitdata <<= size;
    $Dst = result;
}

// i_bs_read_integer_8 Ctx Dst
i_bs_read_integer_8(Ctx, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val($Ctx);
    byte *byte_ptr;
    Uint bit_offset = sb->start % 8;
    Eterm result;

    byte_ptr = erl_sub_bits_get_base(sb) + BYTE_OFFSET(sb->start);
    sb->start += 8;
    result = byte_ptr[0];
    if (bit_offset != 0) {
        result = result << 8 | byte_ptr[1];
        result = ((result << bit_offset) >> 8) & 0xff;
    }
    result = make_small(result);
    $Dst = result;
}

//
// i_bs_get_fixed_integer Ctx Size Flags Dst
//

i_bs_get_fixed_integer := i_bs_get_fixed_integer.fetch.execute;

i_bs_get_fixed_integer.head() {
    Eterm context;
}

i_bs_get_fixed_integer.fetch(Src) {
    context = $Src;
}

i_bs_get_fixed_integer.execute(Size, Flags, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint size = $Size;
    Eterm result;

    LIGHT_SWAPOUT;
    result = erts_bs_get_integer_2(c_p, size, $Flags, sb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);

    $Dst = result;
}

//
// i_get_fixed_binary Ctx Size Dst
//

i_bs_get_fixed_binary := i_bs_get_fixed_binary.fetch.execute;

i_bs_get_fixed_binary.head() {
    Eterm context;
}

i_bs_get_fixed_binary.fetch(Src) {
    context = $Src;
}

i_bs_get_fixed_binary.execute(Size, Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint size = $Size;
    Eterm* htop;
    Eterm result;

    htop = HTOP;
    result = erts_build_sub_bitstring(&htop,
                                      sb->orig & TAG_PTR_MASK__,
                                      (BinRef*)boxed_val(sb->orig),
                                      erl_sub_bits_get_base(sb),
                                      sb->start,
                                      size);
    HTOP = htop;

    sb->start += size;

    $Dst = result;
}

//
// i_get_tail Ctx Dst
//

i_bs_get_tail := i_bs_get_tail.fetch.execute;

i_bs_get_tail.head() {
    Eterm context;
}

i_bs_get_tail.fetch(Src) {
    context = $Src;
}

i_bs_get_tail.execute(Dst) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Eterm* htop;
    Eterm result;

    htop = HTOP;
    result = erts_build_sub_bitstring(&htop,
                                      sb->orig & TAG_PTR_MASK__,
                                      (BinRef*)boxed_val(sb->orig),
                                      erl_sub_bits_get_base(sb),
                                      sb->start,
                                      sb->end - sb->start);
    HTOP = htop;

    $Dst = result;
}

//
// i_bs_skip Ctx Size
//

i_bs_skip := i_bs_skip.fetch.execute;

i_bs_skip.head() {
    Eterm context;
}

i_bs_skip.fetch(Src) {
    context = $Src;
}

i_bs_skip.execute(Size) {
    ErlSubBits *sb = (ErlSubBits*)bitstring_val(context);
    Uint size = $Size;

    sb->start += size;
}

// i_bs_drop Size
i_bs_drop(Size) {
    bitdata <<= $Size;
}
